# Updated by Wei on 2014/07/05
# output the query length distribution for the testing query set
# both for clueweb09B and gov2
from __future__ import division
from operator import itemgetter, attrgetter

import math
import matplotlib
import os
# import pylab
import random
import sys
import time
from sets import Set

print "Program Begins..."
# key: queryLength in int format
# value: freq in int format
queryLengthDistributionDict = {}
# option1
# inputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/100KQueries_head_95K"
# on vidaserver
# for clueweb09B
# inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_queries_ALL_sortedByRandom_testing"
# for gov2
# inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2_tail5K_queries_testing"
# for clueweb09B
# inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/clueweb09B_09-12.topics.polyIRTKCompatible"
# for gov2
inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/gov2_04-06.topics.polyIRTKCompatibleMode"


# for clueweb09B
# outputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_queries_ALL_sortedByRandom_testing_withQueryLengthAdded"
# for gov2
# outputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2_tail5K_queries_testing_withQueryLengthAdded"
# for clueweb09B
# outputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/clueweb09B_09-12.topics.polyIRTKCompatible_withQueryLengthAdded"
# for gov2
outputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/gov2_04-06.topics.polyIRTKCompatibleMode_withQueryLengthAdded"
outputFileHandler = open(outputFileName,"w")

inputFileHandler = open(inputFileName,"r")
currentLine = inputFileHandler.readline()
lineCounter = 1
while currentLine:
    # option1
    data = currentLine.strip().split(":")[1]
    
    # processing option1 (the ORIGINAL one)
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]
    queryContent = data
    
    # processing option2 (the simple split one)
    # queryContent = data
    
    
    queryContentElements = queryContent.strip().split(" ") 
    
    currentQueryTermDict = {}
    for element in queryContentElements:
        if element.strip() != "":
            if element.strip() not in currentQueryTermDict:
                currentQueryTermDict[element.strip()] = 1
    currentQueryLength = len(currentQueryTermDict)
    if currentQueryLength not in queryLengthDistributionDict:
        if currentQueryLength == 0:
            print "lineCounter:",lineCounter,"*",currentLine.strip(),"*"
            # print currentQueryTermDict,"*"
            # print "*",currentLine.strip(),"*"
            # print
        queryLengthDistributionDict[currentQueryLength] = 1
    else:
        if currentQueryLength == 0:
            print "lineCounter:",lineCounter,"*",currentLine.strip(),"*"
            # print currentQueryTermDict,"*"
            # print "*",currentLine.strip(),"*"
            # print
        queryLengthDistributionDict[currentQueryLength] += 1
    outputFileHandler.write(str(currentQueryLength) + " " + currentLine)
    currentLine = inputFileHandler.readline()     
    lineCounter += 1
    # print lineCounter
    if lineCounter % 1000001 == 0:
        print lineCounter
        break

queryLengthList = []
queryLengthList = queryLengthDistributionDict.keys()
queryLengthList.sort(cmp=None, key=None, reverse=False)
# option1
# testing queries
# clueweb09B: 8690
# gov2: 5000
# HJ queries
# clueweb09B: 200
# gov2: 149
TOTAL_NUM_OF_QUERIES = 149
totalQueryLength = 0
averageQueryLength = 0
for queryLength in queryLengthDistributionDict:
    print queryLength,queryLengthDistributionDict[queryLength],queryLengthDistributionDict[queryLength]/TOTAL_NUM_OF_QUERIES
    totalQueryLength += queryLengthDistributionDict[queryLength] * queryLength
averageQueryLength = totalQueryLength / TOTAL_NUM_OF_QUERIES
print "OVERALL:"
print "total # of queries counted:",lineCounter-1
print "totalQueryLength:",totalQueryLength
print "TOTAL_NUM_OF_QUERIES:",TOTAL_NUM_OF_QUERIES
print "average Query Length:",averageQueryLength
print "inputFileName:",inputFileName
print "outputFileName:",outputFileName
print "Program Ends."
inputFileHandler.close()
outputFileHandler.close()

